# -*- coding: utf-8 -*-
import scrapy

import logging
import json
from copy import copy, deepcopy
import random
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class ShantouSpider(scrapy.Spider):
    name = 'shantou'
    allowed_domains = ['14.18.103.122']
    start_urls = ['http://14.18.103.122/']
    project_li_url = 'http://14.18.103.122/mp/NewHouse/getProjectList'  # POST 项目列表页
    project_detail_url = 'http://14.18.103.122/mp/NewHouse/getProjectInfo'  # POST 项目详情页
    building_detail_url = 'http://14.18.103.122/mp/NewHouse/getBuildData'  # POST 楼栋详情页
    room_detail_url = 'http://14.18.103.122/mp/NewHouse/getRoomInfo'  # POST 房号详情页
    flag = False  # 目前考虑到反爬，暂时不进入房号详情页，如需进入，改为True即可

    custom_settings = {
        'DOWNLOAD_DELAY': random.uniform(5, 5.5),
        'CONCURRENT_REQUESTS': 2
    }

    def start_requests(self):
        """
        项目列表首页
        :return:
        """
        data: dict = {
            "CompanyXID": "",
            "HX": "",
            "ID": "",
            "ISMAP": "",
            "LABEL": "",
            "OrderInfo": "1",
            "PageIndex": "1",
            "Region": "",
            "TotalPrice": "",
            "XMMC": "",
            "pageSize": "5"
        }
        yield scrapy.FormRequest(
            self.project_li_url,
            method='POST',
            headers=self.get_headers(),
            formdata=data,
            meta=dict(page_num=1, ),
        )

    def parse(self, response):
        """
        项目列表页
        :param response:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        try:
            resp_dict = json.loads(response.body.decode())
            current_page_num = int(resp_dict['Data'].get('PageIndex', None))
            assert current_page_num == page_num, f'第{page_num}页，页数编号错误，page_num={current_page_num}'
            project_li = resp_dict['Data'].get('DataList', list())
            assert project_li, f'第{page_num}页，项目列表获取为空'
            total_page_num = int(resp_dict['Data'].get('PageCount', None))
            assert total_page_num, f'第{page_num}页，总页数获取出错'
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error(f'第{page_num}页，项目列表获取出错，msg:{e if e else "error"}')
        else:
            # 遍历，提取项目列表信息
            for project_dict in project_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = project_dict.get('XMMC', None)  # 项目名称
                item_eg['projectId'] = project_dict.get('XID', None)  # 项目id
                item_eg['planningUse'] = project_dict.get('GHYT', None)  # 规划用途
                item_eg['districtName'] = project_dict.get('SZQY', None)  # 项目区域
                item_eg['projectAddress'] = project_dict.get('JYDZ', None)  # 项目地址
                item_eg['developerName'] = project_dict.get('COMPANYNAME', None)  # 开发商
                item_eg['soldRoomNo'] = project_dict.get('YSTS', None)  # 已售套数
                item_eg['unsoldRoomNo'] = project_dict.get('WSTS', None)  # 未售套数
                item_eg['openingDate'] = project_dict.get('ZJKP', None)  # 开盘日期

                # 获取项目详情
                if item_eg['projectId']:
                    data: dict = {
                        'id': item_eg['projectId'],
                    }
                    yield scrapy.FormRequest(
                        self.project_detail_url,
                        headers=self.get_headers(),
                        formdata=data,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=20,
                    )
                else:
                    logger.error(f"第{page_num}页-{item_eg['projectName']}，项目id获取为空，无法获取项目详情")

            # 翻页
            data: dict = {
                "CompanyXID": "",
                "HX": "",
                "ID": "",
                "ISMAP": "",
                "LABEL": "",
                "OrderInfo": "1",
                "Region": "",
                "TotalPrice": "",
                "XMMC": "",
                "pageSize": "5"
            }
            for next_page_num in range(2, total_page_num + 1):
                data['PageIndex'] = str(next_page_num)
                yield scrapy.FormRequest(
                    self.project_li_url,
                    method='POST',
                    headers=self.get_headers(),
                    formdata=data,
                    meta=dict(page_num=next_page_num, ),
                    priority=10,
                )

    def parse_project_detail(self, response):
        """
        项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            data_dict: dict = json.loads(response.body.decode()).get('Data', dict())
            assert data_dict, f'{item_eg["projectName"]} 项目详情信息获取为空'
            base_info: dict = data_dict.get('BaseInfo', dict())
            land_p: list = data_dict.get('TDZ', list())
            construction_p: list = data_dict.get('JSGCSGXKZ', list())
            planning_p: list = data_dict.get('JSGCGHXKZ', list())
            permit_li: list = data_dict.get('XKZ', list())
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error(f'{item_eg["projectName"]} 项目详情信息获取出错')
        else:
            item_eg['qualificationCertificateNo'] = base_info.get('ZZZSBH', None)  # 资质证书编号
            item_eg['qualificationGrade'] = base_info.get('ZZDJ', None)  # 资质等级
            item_eg['estateTotalArea'] = base_info.get('JZMJ', None)  # 建筑总面积
            item_eg['floorAreaRatio'] = base_info.get('RJL', None)  # 容积率
            item_eg['buildingDensity'] = base_info.get('JZMD', None)  # 建筑密度
            item_eg['greeningRate'] = base_info.get('LDV', None)  # 绿化率
            item_eg['latBd'] = base_info.get('LATITUDE', None)  # 经度
            item_eg['lngBd'] = base_info.get('LONGITUDE', None)  # 纬度
            item_eg['landPermit'] = [i.get('ZH', None) for i in land_p if i] if land_p else None  # 土地许可证
            item_eg['constructionPermit'] = [i.get('ZH', None) for i in construction_p if
                                             i] if construction_p else None  # 施工许可证
            item_eg['planningPermit'] = [i.get('ZH', None) for i in planning_p if i] if planning_p else None  # 规划许可证
            if permit_li:
                build_filter_dict = dict()  # 用于重复楼栋的处理
                # 遍历，获取预售许可证信息
                for permit_dict in permit_li:
                    item_eg['preSalePermit'] = permit_dict.get('XKZH', None)  # 预售许可证
                    item_eg['certDate'] = permit_dict.get('FZRQ', None)  # 发证日期
                    item_eg['certStartDate'] = permit_dict.get('YXQX1', None)  # 有效期起
                    item_eg['certExpiryDate'] = permit_dict.get('YXQX2', None)  # 有效期止
                    item_eg['approvalRoomNo'] = permit_dict.get('YSZTS', None)  # 批准套数
                    item_eg['approvalArea'] = permit_dict.get('YSZMJ', None)  # 批准面积
                    item_eg['preSaleBuilding'] = permit_dict.get('BindBuild', None)  # 预售楼栋
                    item_eg['permitResidenceNum'] = permit_dict.get('ZZTS', None)  # 许可证住宅套数
                    item_eg['permitResidenceArea'] = permit_dict.get('ZZMJ', None)  # 许可证住宅面积
                    item_eg['permitCommerceNum'] = permit_dict.get('SYTS', None)  # 许可证商业套数
                    item_eg['permitCommerceArea'] = permit_dict.get('SYMJ', None)  # 许可证商业面积
                    item_eg['permitOtherNum'] = permit_dict.get('QTTS', None)  # 许可证其他套数
                    item_eg['permitOtherArea'] = permit_dict.get('QTMJ', None)  # 许可证其他面积
                    item_eg['permitOfficialNum'] = permit_dict.get('BGTS', None)  # 许可证办公套数
                    item_eg['permitOfficialArea'] = permit_dict.get('BGMJ', None)  # 许可证办公面积
                    item_eg['permitParkingSpacesNum'] = permit_dict.get('CKTS', None)  # 许可证车位套数
                    item_eg['permitParkingSpacesArea'] = permit_dict.get('CKMJ', None)  # 许可证车位面积
                    yield item_eg

                    build_li: list = permit_dict.get('BUILDLIST', list())
                    if build_li:
                        # 遍历，提取楼栋列表信息
                        for build_dict in build_li:
                            build_id = build_dict['BLDID']
                            bd_dict = build_filter_dict.get(build_id, dict())
                            bd_dict['projectName'] = item_eg['projectName']  # 项目名称
                            bd_dict['projectId'] = item_eg['projectId']  # 项目id
                            bd_dict['buildingId'] = build_id  # 楼栋id
                            bd_dict['blockName'] = build_dict.get('ZMC', None)  # 楼栋名字
                            bd_dict['handoverDate'] = build_dict.get('JFRQ', None)  # 交房日期
                            bd_dict['floorTotalNo'] = build_dict.get('ZCS', None)  # 总层数
                            bd_dict['floorOverGroundNo'] = build_dict.get('DSCS', None)  # 地上层数
                            bd_dict['floorUnderGroundNo'] = build_dict.get('DXCS', None)  # 地下层数
                            bd_dict['totalRoomNo'] = build_dict.get('HS', None)  # 总套数
                            bd_dict['forecastBuildArea'] = build_dict.get('YCJZMJ', None)  # 预测建筑面积
                            bd_dict['realBuildArea'] = build_dict.get('SCJZMJ', None)  # 实测建筑面积
                            bd_dict['buildingStructure'] = build_dict.get('JZJG', None)  # 建筑结构
                            bd_dict['preSalePermit'] = bd_dict.get('preSalePermit', list()) + [
                                item_eg['preSalePermit']]  # 预售许可证
                            build_filter_dict[build_id] = bd_dict
                    else:
                        logger.warning('{}-{} 楼栋列表获取为空'.format(item_eg['projectName'], item_eg['preSalePermit']))

                # 遍历，实例化楼栋item
                for bd_id, bd_dict in build_filter_dict.items():
                    item_bd = FdcBuildingItem()
                    for key, value in bd_dict.items():
                        item_bd[key] = value
                    # 构造POST请求，获取楼栋详情(房号列表)
                    data: dict = {
                        'param': {"buildid": bd_id,
                                  "where": [{"grouprid": "129", "value": ""}, {"grouprid": "133", "value": ""},
                                            {"grouprid": "10", "value": ""}, {"grouprid": "862", "value": ""},
                                            {"grouprid": "864", "value": ""}]},
                        'fileName': 'publicbuildtable.config',
                    }
                    data['param'] = json.dumps(data['param'])
                    yield scrapy.FormRequest(
                        self.building_detail_url,
                        headers=self.get_headers(),
                        formdata=data,
                        callback=self.parse_building_detail,
                        meta=dict(item_bd=deepcopy(item_bd), ),
                        priority=30,
                    )
            else:
                logger.warning('{} 预售许可证列表获取为空'.format(item_eg['projectName']))

    def parse_building_detail(self, response):
        """
        楼栋详情(房号列表)
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            resp_dict = json.loads(response.body.decode())
            lpd_data: dict = resp_dict['Data'].get('lpbData', dict())
            assert lpd_data, f'{item_bd["projectName"]}-{item_bd["blockName"]} 楼栋详情信息为空'
            bd_dict: dict = lpd_data.get('build', dict())
            room_li: list = lpd_data.get('room', list())
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error(f'{item_bd["projectName"]}-{item_bd["blockName"]} 楼栋详情获取出错')
        else:
            # 提取楼栋信息
            item_bd['buildingLocation'] = bd_dict.get('ZZL', None)
            yield item_bd

            # 遍历，提起房号列表信息
            for room_dict in room_li:
                item_rm = FdcRoomItem()
                item_rm['projectName'] = item_bd['projectName']  # 项目名称
                item_rm['projectId'] = item_bd['projectId']  # 项目id
                item_rm['buildingId'] = item_bd['buildingId']  # 楼栋id
                item_rm['blockName'] = item_bd['blockName']  # 楼栋名字
                item_rm['roomNo'] = room_dict.get('HH', None)  # 房号
                item_rm['roomId'] = room_dict.get('FWID', None)  # 房屋id
                item_rm['roomStructure'] = room_dict.get('HX', None)  # 户型
                item_rm['unitNo'] = room_dict.get('DYH', None)  # 单元号
                item_rm['roomUse'] = room_dict.get('FWYT', None)  # 房屋用途
                item_rm['roomLocation'] = room_dict.get('FWZL', None)  # 房屋坐落
                item_rm['saleStatus'] = room_dict.get('FWZT', None)  # 销售状态
                item_rm['buildingArea'] = room_dict.get('JZMJ', None)  # 建筑面积
                item_rm['apportionmentArea'] = room_dict.get('FTMJ', None)  # 分摊面积
                item_rm['innerArea'] = room_dict.get('TNMJ', None)  # 套内面积
                item_rm['nominalFloor'] = room_dict.get('MYC', None)  # 名义层
                item_rm['physicsFloor'] = room_dict.get('SJC', None)  # 物理层
                item_rm['recordTotalPrice'] = room_dict.get('WJBAZJ', None)  # 备案总价

                # 构造POST请求，获取房号详情
                if self.flag:
                    if item_rm['roomId']:
                        data: dict = {
                            'FWID': item_rm['roomId']
                        }
                        yield scrapy.FormRequest(
                            self.room_detail_url,
                            headers=self.get_headers(),
                            formdata=data,
                            callback=self.parse_room_detail,
                            meta=dict(item_rm=deepcopy(item_rm), ),
                            priority=40,
                        )
                    else:
                        logger.error('{}-{}-{} 房号id提取为空，无法获取房号详情'.format(item_rm['projectName'], item_rm['blockName'],
                                                                         item_rm['roomNo']))
                else:
                    yield item_rm

    def parse_room_detail(self, reponse):
        """
        房号详情
        :param reponse:
        :return:
        """
        item_rm = copy(reponse.meta['item_rm'])
        try:
            resp_dict = json.loads(reponse.body.decode())
            data_dict = resp_dict.get('Data', dict())
            assert data_dict, '{}-{}-{} 房号详情获取为空'.format(item_rm['projectName'], item_rm['blockName'],
                                                         item_rm['roomNo'])
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error('{}-{}-{} 房号详情获取出错'.format(item_rm['projectName'], item_rm['blockName'], item_rm['roomNo']))
        else:
            item_rm['recordTotalPrice'] = data_dict.get('WJBAZJ', None)  # 备案总价
            yield item_rm

    def get_headers(self) -> dict:
        """
        获取请求头
        :return:
        """
        headers: dict = {
            'Referer': self.start_urls[0],
            'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
            'ClientId': 'ihouse',
            'ClientUserId': '',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Accept': 'application/json, text/plain, */*',
        }
        return headers
